
# File management libraries
import pickle as pkl

# Utilities
import os
from tqdm import tqdm
from typing import Union
from collections import defaultdict
from operator import itemgetter

# Web scraping libraries
import requests
from bs4 import BeautifulSoup

# Data manipulation libraries
import pandas as pd
import numpy as np

# Network analysis libraries
import networkx as nx
import netwulf as nw
from community import community_louvain

# Text processing libraries
import re
import pyphen
import nltk
from nltk import tokenize
from nltk.corpus import stopwords
from nltk import word_tokenize
from nltk.sentiment.vader import SentimentIntensityAnalyzer
# nltk.download(["vader_lexicon"])

# Plotting libraries
import matplotlib.pyplot as plt
from wordcloud import WordCloud, ImageColorGenerator
from PIL import Image

colors = ['#ad262d', '#013d7b', '#e2a226', '#007447', '#4a2222', '#b37a35', '#948ba0', '#dc7d69', '#f6dcb7']

# Custom libraries
from Utils.Swap import ex_modularity
from Utils.get_text import *

# Flatten list function
def flatten(list_list : list) -> list:
    return [item for sublist in list_list for item in sublist]


len_books = []

book_titles = ["Book 1 - The Philosopher's Stone",
               "Book 2 - The Chamber of Secrets",
               "Book 3 - The Prisoner of Azkaban",
               "Book 4 - The Goblet of Fire",
               "Book 5 - The Order of the Phoenix",
               "Book 6 - The Half Blood Prince",
               "Book 7 - The Deathly Hallows"]

for title in book_titles:
    book = open("Data/OriginalBooks/" + title + ".txt", "r").read()
    len_books.append((len(book.split(" ")), len(book)))

print(f"The raw-book dataset consists of {len(len_books)} books, with {sum([l[0] for l in len_books])} words, and {sum([l[1] for l in len_books])} characters.")
df_characters_raw = pd.read_csv("Data/CharacterData.csv")
print(f"The raw character-list for all 7 books consisted of {len(df_characters_raw)} characters.")

The raw-book dataset consists of 7 books, with 1168851 words, and 6765119 characters.
The raw character-list for all 7 books consisted of 707 characters.


def split_chapter(Book):
    split_string = "\n{9}|\n{8}|\n{7}|\n{6}|\n{5}[0-9]+\s?\n{5}"
    Chapters_text = {idx+1 : chapter for idx, chapter in enumerate(re.split(split_string, Book[7:]))}

    return Chapters_text

def split_pages(Chapter,split_string):
    Pages_text = re.split(split_string,Chapter)
    return Pages_text

def format_page(Page):
    Page = re.sub("\n+|\t","",Page)
    Page = tokenize.sent_tokenize(Page)
    return Page

def split_book(Book,split_string):
    Chapters_text = split_chapter(Book)
    for chapter in Chapters_text:
        Pages_text = split_pages(Chapters_text[chapter],split_string)
        Pages_text = [format_page(Page) for Page in Pages_text[:-1]]
        Chapters_text[chapter] = Pages_text

    for chapter in Chapters_text:
        for idx,page in enumerate(Chapters_text[chapter]):
            if idx == 0:
                continue
            else:
                if not page[0][0].isupper():

                    Chapters_text[chapter][idx-1][-1] = Chapters_text[chapter][idx-1][-1] + " " + page[0]
                    
                    Chapters_text[chapter][idx] = page[1:]
                
    return Chapters_text

book_titles = ["Book 1 - The Philosopher's Stone",
               "Book 2 - The Chamber of Secrets",
               "Book 3 - The Prisoner of Azkaban",
               "Book 4 - The Goblet of Fire",
               "Book 5 - The Order of the Phoenix",
               "Book 6 - The Half Blood Prince",
               "Book 7 - The Deathly Hallows"]

split_string = [
    "Page \| [0-9]+ Harry Potter and the Philosophers Stone -\s?J.K. Rowling",
    "Page \| [0-9]+ Harry Potter and the Chamber of Secrets -\s?J.K. Rowling",
    "Page \| [0-9]+ Harry Potter and the Prisoner of Azkaban -\s?J.K. Rowling",
    "P\s?a\s?g\s?e\s?\|\s?[0-9]+\s?\n*\s?Harry Potter and the Goblet of Fire -\s?J.K. Rowling",
    "P\s?a\s?g\s?e\s?\|\s?[0-9lUO]+\s?\n*\s*Harry Potter and the Order of the Phoenix -\s?J.K. Rowling",
    "P\s?a\s?g\s?e\s?\|\s?[0-9lUO]+\s?\n*\s*Harry Potter and the Half Blood Prince -\s?J.K. Rowling",
    "P\s?a\s?g\s?e\s?\|\s?[0-9lUO]+\s?\n*\s*Harry Potter and the Deathly Hallows -\s?J.K. Rowling"
]

for book_nr, (book_title, split) in enumerate(zip(book_titles, split_string)):
    # Get raw book text
    with open(f"Data/OriginalBooks/{book_title}", encoding="utf8") as f:
        book = f.read()

    # Split book using the function into a dictionary
    book_dict = split_book(book)

    # Save book_dict as pickle file
    with open(f"Data/Books formatted/Book{book_nr + 1}.pkl", "wb") as filename:
        pkl.dump(book_dict, filename)


def get_edge_alias(character: str = 'Harry Potter', alias_list: list = []):
    with open("Temp/character_aliases_edgecases.pkl", "rb") as f:
        edge_aliases = pkl.load(f)

    if character in edge_aliases.keys():
        alias_list.append(edge_aliases[character])
    
    return alias_list

def add_first_name(character: str = 'Harry Potter', alias_list: list = []):
    none_aliases = ['A', 'Mr', 'Mrs', 'Dr', 'Manager', 'The', 'Father','Sorting', 'wizard', 'Chancellor', 'Workmen', 'Waitress', 'Sir',
                    'Nearly-Headless', 'Fat','Aged', 'Blood-Sucking', 'Forbidden','Unidentified', 'Zoo', 'Kepper', 'Muggle', 'Muggle-Born',
                    'Senior', 'Junior', 'Board', 'Committee','Academy', 'Ministry', 'Department',
                    'Little', 'Great', 'Old', 'Young', 'Head', 'Headmaster', 'Headmistress','Weird','Care', 'Montgomery', 'Hogwarts', 'Frank']
    
    none_name = ["'s", "s'", "family"]

    if ((character.split(' ')[0] not in none_aliases) and (all([s not in character for s in none_name])) and (character.split(' ')[0].isalpha())):
        alias_list.append(character.split(' ')[0])
    
    return alias_list

def get_aliases(character: str = 'Harry Potter', link: str = '/wiki/Harry_Potter', df: pd.DataFrame = pd.DataFrame()):
    query = requests.get(f"https://harrypotter.fandom.com/api.php?action=parse&page={link.split('/')[-1].split('#')[0]}&format=json").json()
    HTML = query['parse']['text']['*']
    soup = BeautifulSoup(HTML, 'html.parser')

    # Get aliases
    try:
        # Character has aliases
        alias_ = soup.find(string='Also known as').findNext("div")
        if alias_.find_all('li') != []:
            # Character has multiple aliases
            alias_ = [i.get_text() for i in alias_.find_all('li')]
            alias_ = [re.split('\s?(\()|(\[)',j)[0] for j in alias_]
        else:
            # Character has only one alias
            alias_ = [re.split('\s?(\()|(\[)',alias_.get_text())[0]]
    except:
        # Character has no aliases
        alias_ = []

    # Ensure that aliases are not the same as a character name
    alias_ = [i for i in alias_ if not df['Name'].str.contains(f'{i}', regex=False).any()]

    # Add edge cases and first name
    alias_ = get_edge_alias(character = character, alias_list = alias_)
    alias_ = add_first_name(character = character, alias_list = alias_)

    # Ensure that aliases are unique and not the same as the character's name
    alias_ = list(set(alias_))
    alias_ = [i for i in alias_ if i != character]

    return alias_


def remove_alias_same_as_name(alias_list: list = [], character_Names: list = []):
    # Remove aliases that are the same as character names
    alias_list = [i for i in alias_list if i not in character_Names]

    return alias_list

def add_name(alias_list: list = [], character_name: str = ''):
    # Add character name to alias list
    alias_list.append(character_name)
    list(set(alias_list))
    return alias_list

def remove_dublicate_aliases(alias_list: list = [], all_previous_character_aliases: list = []):
    # Remove duplicate aliases
    alias_list = [i for i in alias_list if i not in all_previous_character_aliases]
    return alias_list

def remove_dublicates(character_df: pd.DataFrame, remove_character: list = []):
    # remove duplicate characters
    for character in remove_character:
        character_df = character_df[character_df['Name'] != character]

    character_df.reset_index(drop=True, inplace=True)

    for idx, character in enumerate(character_df['Name']):
        character_df.loc[idx, 'Aliases'] = remove_alias_same_as_name(alias_list = character_df.loc[idx, 'Aliases'],
                                                                     character_Names = character_df['Name'].tolist())
    
        character_df.loc[idx, 'Aliases'] = add_name(alias_list = character_df.loc[idx, 'Aliases'], character_name = character_df.loc[idx, 'Name'])
    
    for i in range(len(character_df), -1, -1):
        aliases = [alias for aliases in character_df.loc[:i, 'Aliases'] for alias in aliases]
        character_df.loc[i, 'Aliases'] = remove_dublicate_aliases(alias_list = character_df.loc[i, 'Aliases'],
                                                                  all_previous_character_aliases = aliases)
                                                                
    return character_df

# Loading the raw character list
character_df = pd.read_csv("Data/CharacterData.csv")

# Getting the aliases for each character
aliases = [get_aliases(character = i, link = j, df = character_df) for i,j in tqdm(zip(character_df['Name'], character_df['Link']))]
character_df['Aliases'] = aliases

# Optionally save the character_df as CharacterWikis.csv
character_df.to_csv("Data/CharacterWikis.csv", index = False)

# Optionally load the character_df as CharacterWikis.csv
character_df = pd.read_csv("Data/CharacterWikis.csv")
character_df.Aliases = [eval(char) for char in character_df.Aliases]

# Load the potential dublicates text file
with open("Data/potential_dublicates.txt","r") as f:
    potential_dublicates = f.readlines()
    potential_dublicates = [char.strip() for char in potential_dublicates]
    remove_char = [char[:-4] for char in potential_dublicates if char[-3:] == "(R)"]

# Remove the dublicates
character_df = remove_dublicates(character_df, potential_dublicates)

# Save the character_df as CharacterWikis_clean.csv
character_df.to_csv("Data/CharacterWikis_clean.csv", index = False)


# Load formatted book (dict of chapters with list of lists of pages and sentences)
initial_character_list = pd.read_csv("Data/CharacterWikis.csv")
final_character_list = pd.read_csv("Data/CharacterWikis_clean.csv")
final_character_list.Aliases = [eval(char) for char in final_character_list.Aliases]

print(f"The initial character list has {len(initial_character_list)} characters, and {sum([len(eval(l)) for l in initial_character_list['Aliases']]) + len(initial_character_list)} aliases in total.")
print(f"The final character list after data cleaning has {len(final_character_list)} characters, and {sum([len(l) for l in final_character_list.Aliases])} aliases in total.")

The initial character list has 707 characters, and 1393 aliases in total.
The final character list after data cleaning has 677 characters, and 1323 aliases in total.


def get_edge_chapter_weight(character1: list, character2: list, chapter: list):
    weight = 0
    for page in chapter:
        page_text = ' '.join(page)
        if any([char in page_text for char in character1]) and any([char in page_text for char in character2]):
            weight += 1
    
    return weight

def get_edge_book_weight(character1: list, character2: list, book: dict):
    weight = defaultdict(int)
    for chapternr in range(1, len(book)+1):
        weight[chapternr] = get_edge_chapter_weight(character1, character2, book[chapternr])
    return weight

def get_node_size(character: list, book: dict):
    size = defaultdict(int)
    for chapternr, chapter  in book.items():
        for page in chapter:
            page_text = ' '.join(page)
            if any([char in page_text for char in character]):
                size[chapternr] += 1
    return size

Book_networks = {}

for book_nr in range(1,8):
    # Load formatted book (dict of chapters with list of lists of pages and sentences)
    with open(f"Data/Books formatted/Book{book_nr}.pkl","rb") as f:
        book = pkl.load(f)

    print(f"Creating book graph for book {book_nr}...")

    # Draw network
    G = nx.Graph()

    # Add nodes to graph
    for character in tqdm(characters, desc='Adding character nodes to graph...'):
        character['sizes'] = get_node_size(character['aliases'], book)
        G.add_node(character['name'], sizes=character['sizes'], attr=character['aliases'])

    # Add edges to graph
    for i, char1 in enumerate(tqdm(characters, desc='Adding edges to graph...')):
        for j, char2 in enumerate(characters[i+1:]):
            edge_weight = get_edge_book_weight(char1['aliases'], char2['aliases'], book)
            if sum(edge_weight.values()) != 0:
                G.add_edge(char1['name'], char2['name'], weight=edge_weight)

    # Save graph
    Book_networks[book_nr] = G.copy()


def text_page(char : list, page : list) -> list:
    text = []
    for i,line in enumerate(page):
        if any(alias in line for alias in char):
            if i == 0:
                try:
                    text.append([page[i], page[i+1]])
                except:
                    text.append([page[i]])
            elif i == len(page)-1:
                text.append([page[i-1], page[i]])
            else:
                text.append([page[i-1], page[i], page[i+1]])
            
    return text

def text_chapter(char : list, chapter : list) -> list:
    text = []
    for page in chapter:
        text.append(text_page(char, page))
    
    return text

def tokenize(text):
    # Exclude punctuation and numbers and set everything to lower case and exclude stopwords
    text = word_tokenize(text)
    text = [w.lower() for w in text if w.isalpha()]
    text = [w for w in text if not w.lower() in set(stopwords.words('english'))]

    return text

def tokenize_text(text : list) -> str:
    text_tokenized = []

    for page in text:
        for finding in page:
            for sentence in finding:
                text_tokenized.extend(tokenize(sentence))
    
    return text_tokenized

def text_book(char : list, book : list) -> dict:
    text = {}
    for chapnr,chapter in book.items():
        text[chapnr] = tokenize_text(text_chapter(char, chapter))
    
    return text

characters_df = pd.read_csv("Data/CharacterWikis_clean.csv")

characters_dynamic_text = {}

for char in tqdm(characters_df):
    char_book_text = {}

    for book_nr in range(1,8):
        # Load formatted book (dict of chapters with list of lists of pages and sentences)
        with open(f"Data/Books formatted/Book{book_nr}.pkl","rb") as f:
            book = pkl.load(f)

        char_book_text[book_nr] = text_book(char['aliases'], book)

    characters_dynamic_text[char['name']] = char_book_text

with open(f"Temp/Dynamic_text.pkl","wb") as f:
    pkl.dump(Characters_dynamic_text,f)


# Network
with open("Temp/Book_networks.pkl", "rb") as f:
    book_networks = pkl.load(f)

# Text
with open("Temp/Dynamic_text.pkl", "rb") as f:
    book_text = pkl.load(f)


def getCentralization(centrality):
    n_val = float(len(centrality))
    c_denominator = (n_val-1)*(n_val-2)
    
    c_node_max = max(centrality.values())
    c_sorted = sorted(centrality.values(), reverse=True)
    c_numerator = sum([(c_node_max*(n_val-1) - value*(n_val-1)) for value in c_sorted])
    
    return float(c_numerator/c_denominator)

sizes = np.array([len([n for s in book.nodes.data('sizes') for n in s[1] if n != 0]) for book in book_networks.values()])
sizes_w = np.array([sum([sum(s[1].values()) for s in book.nodes.data('sizes')]) for book in book_networks.values()])

edges = np.array([book.number_of_edges() for book in book_networks.values()])
edges_w = np.array([sum([sum(w[2].values()) for w in book.edges.data('weight')]) for book in book_networks.values()])

centralitization = np.array([getCentralization(nx.degree_centrality(book)) for book in book_networks.values()])

fig, ax = plt.subplots(1, 2, figsize=(15,5), constrained_layout=True)
ax[0].plot(sizes, label = "Number of nodes (Number of Characters)", color = colors[0], alpha = 0.5)
ax[0].plot(sizes_w, label = "Sum of weighted nodes (Number of Character apperances)", color = colors[0])
ax0_2 = ax[0].twinx()

ax0_2.plot(edges, label = "Number of edges (Number of Relationships)", color = colors[1], alpha = 0.5)
ax0_2.plot(edges_w, label = "Sum of weighted edges (Number of Interactions)", color = colors[1])
ax[0].set_xlabel("Book")
ax[0].set_ylabel("# Nodes")
ax0_2.set_ylabel("# Edges")

lines, labels = ax[0].get_legend_handles_labels()
lines2, labels2 = ax0_2.get_legend_handles_labels()
ax0_2.legend(lines + lines2, labels + labels2, bbox_to_anchor=(0., -0.3, 1., .102), loc='lower left',
              ncol=1, mode="expand", borderaxespad=0.)

ax[1].plot((1*edges)/(sizes*(sizes-1)), label = "Density (Narrative style: how divided are subcommunities in the story?)", color = colors[2])
ax[1].plot((1*edges_w)/(sizes_w*(sizes_w-1)), label = "Weighted Density", color = colors[2], alpha = 0.5)
ax1_2 = ax[1].twinx()
ax1_2.plot(centralitization, label = "Centralization (Narrative style: how much is the story focused on a few characters?)", color = colors[3])
ax[1].set_ylabel("Density")
ax1_2.set_ylabel("Centralization")

lines, labels = ax[1].get_legend_handles_labels()
lines2, labels2 = ax1_2.get_legend_handles_labels()
ax1_2.legend(lines + lines2, labels + labels2, bbox_to_anchor=(0., -0.3, 1., .102), loc='lower left',
              ncol=1, mode="expand", borderaxespad=0.)

fig.suptitle("Network properties over the books")
plt.show()


fig, ax = plt.subplots(1, 2, figsize=(15,5), constrained_layout=True)

x_ticks = np.cumsum([(len(book_text['Harry Potter'][book_nr])) for book_nr in range(1,8)])

for i, character in enumerate(['Harry Potter', 'Hermione Granger', 'Ron Weasley', 'Albus Dumbledore', 'Lord Voldemort', 'Dudley Dursley']):
    number_of_words = []
    number_of_sentences = []
    for book_nr in range(1,8):
        number_of_words.extend([len(wordlist['tokens']) for wordlist in book_text[character][book_nr].values()])
        number_of_sentences.extend([len(sentences['text'])*3 for sentences in book_text[character][book_nr].values()])
    
    ax[0].plot(number_of_words, label = f"Number of words for {character}", color = colors[i])
    ax[1].plot(number_of_sentences, label = f"Number of sentences {character}", color = colors[i])

    ax[0].set_ylabel("# Words")
    ax[1].set_ylabel("# Sentences")

    for j in range(2):
        ax[j].set_xlabel("Book")
        ax[j].set_xticks([])
        ax[j].set_xticks(x_ticks, labels=[f"Book {i}" for i in range(1,8)])
        ax[j].legend(bbox_to_anchor=(0., -0.35, 1., .102), loc='lower left',
                    ncol=2, mode="expand", borderaxespad=0.)

fig.suptitle("Text properties over the books")
plt.show()


def get_louvain_partitioning(G):
    partition = community_louvain.best_partition(G)

    # Add partition to graph
    for node in G.nodes:
        G.nodes[node]['group'] = partition[node]
    
    return G, partition


def get_network_state(book_nr, chapter_nr, book_only = False, chapter_from = None):

    # Number of chapters per book
    n_chapters = {1 : 17, 2 : 18, 3 : 22, 4 : 37, 5 : 38, 6 : 30, 7 : 37}

    # Check for valid input
    if book_nr not in range(1,8):
        raise ValueError("book_nr must be between 1 and 7")
    if chapter_nr not in range(1, n_chapters[book_nr]+1):
        raise ValueError(f"chapter_nr must be between 1 and {n_chapters[book_nr]} for book {book_nr}")

    # Add nodes and edges to G_chapter
    G_chapter = nx.Graph()
    G_chapter.add_nodes_from([(node[0], {'size': node[1]['sizes'][chapter_nr], 'attr': node[1]['attr']})
                            for node in book_networks[book_nr].nodes(data=True) if node[1]['sizes'][chapter_nr] != 0])

    G_chapter.add_weighted_edges_from([(edge[0], edge[1], edge[2]['weight'][chapter_nr])
                                        for edge in book_networks[book_nr].edges(data=True) if edge[2]['weight'][chapter_nr] != 0])
    
    G_chapter, partition_chapter = get_louvain_partitioning(G_chapter)

    # Add all characters to graph G
    G = nx.Graph()
    G.add_nodes_from([(c.Name, {'attr': c.Aliases}) for c in final_character_list.itertuples()], size = 0)

    # Update node sizes and edge-weights for G
    if book_only == False:
        G.add_nodes_from([(c.Name, {'size': sum([sum(book_networks[book].nodes[c.Name]['sizes'].values()) for book in range(1, book_nr)])})
                           for c in final_character_list.itertuples()])
        
        for book in range(1, book_nr):
            for node1, node2, weight in book_networks[book].edges.data('weight'):
                if G.has_edge(node1, node2):
                    G.edges[node1, node2]['weight'] += sum(weight.values())
                else:
                    G.add_edge(node1, node2, weight=sum(weight.values()))
        
        for character in final_character_list.itertuples():
                G.nodes[character.Name]['size'] += sum(book_networks[book_nr].nodes[character.Name]['sizes'][i] for i in range(1, chapter_nr + 1))
        
        for node1, node2, weight in book_networks[book_nr].edges.data('weight'): 
            if sum(list(weight.values())[1:chapter_nr+1]) != 0:
                if G.has_edge(node1, node2):
                    G.edges[node1, node2]['weight'] += sum(list(weight.values())[1:chapter_nr+1])
                else:
                    G.add_edge(node1, node2, weight = sum(list(weight.values())[1:chapter_nr+1]))

    else:
        
        if chapter_from == None:
            chapter_from = 1
        
        for character in final_character_list.itertuples():
                G.nodes[character.Name]['size'] += sum(book_networks[book_nr].nodes[character.Name]['sizes'][i] for i in range(chapter_from, chapter_nr + 1))
        
        for node1, node2, weight in book_networks[book_nr].edges.data('weight'): 
            if sum(list(weight.values())[chapter_from:chapter_nr+1]) != 0:
                if G.has_edge(node1, node2):
                    G.edges[node1, node2]['weight'] += sum(list(weight.values())[chapter_from:chapter_nr+1])
                else:
                    G.add_edge(node1, node2, weight = sum(list(weight.values())[chapter_from:chapter_nr+1]))

    # Get largest connected component
    G = G.subgraph(max(nx.connected_components(G), key=len)).copy()
    G, partition = get_louvain_partitioning(G)

    return G, G_chapter, partition, partition_chapter


def get_random_network(n_nodes, n_edges):
    # Create a random graph

    p = 2/((n_nodes-1)*n_nodes) * n_edges
    G_random = nx.erdos_renyi_graph(n_nodes, p)
    largest_cc = max(nx.connected_components(G_random), key=len)
    G_random_sub = G_random.subgraph(largest_cc).copy()

    return G_random_sub


def get_stats_for_random(G_random, N_random = 20):
    temp_Degree = np.array(list(map(nx.degree_assortativity_coefficient, G_random)))
    temp_Clustering = np.array(list(map(nx.average_clustering, G_random)))
    temp_Density = np.array(list(map(nx.density, G_random)))
    temp_Shortest_path = np.array(list(map(nx.average_shortest_path_length, G_random)))

    random_stats = {"Degree": (temp_Degree.mean(), temp_Degree.std()/np.sqrt(N_random)),
                    "Clustering": (temp_Clustering.mean(),temp_Clustering.std()/np.sqrt(N_random)),
                    "Density": (temp_Density.mean(),temp_Density.std()/np.sqrt(N_random)),
                    "Shortest_path": (temp_Shortest_path.mean(),temp_Shortest_path.std()/np.sqrt(N_random))}

    return random_stats

def get_stats_for_network(G, partition):
    Modularity = community_louvain.modularity(partition, G)
    Degree = nx.degree_assortativity_coefficient(G)
    Clustering = nx.average_clustering(G)
    Density = nx.density(G)
    try : 
        Shortest_path = nx.average_shortest_path_length(G)
    except:
        Shortest_path = None

    network_stats = {"Modularity": Modularity, "Degree": Degree, "Clustering": Clustering, "Density": Density, "Shortest_path": Shortest_path}

    return network_stats

def get_network_centrality(G):
    return nx.degree_centrality(G)

def get_overall_network_stats(book_nr, chapter_nr, N_random = 20):

    G, G_chap, part, part_chap = get_network_state(book_nr, chapter_nr)
    n_nodes, n_edges = G.number_of_nodes(), G.number_of_edges()

    G_random = [get_random_network(n_nodes, n_edges) for _ in range(N_random)]

    return (get_stats_for_random(G_random, N_random), get_stats_for_network(G, part),
    get_stats_for_network(G_chap, part_chap), get_network_centrality(G), get_network_centrality(G_chap))


def plot_network_stat(stat, stat_chapter, stat_random, title, ylabel):

    fig, ax = plt.subplots(figsize=(10,5))
    ax.plot(stat, label=f"{ylabel} (Accumulated)", color = colors[2])
    ax.plot(stat_chapter, label=f"{ylabel} (chapter)", color = colors[1])
    
    # plot mean
    if stat_random is not None:
        mean = np.array([x[0] for x in stat_random])
        std = np.array([x[1] for x in stat_random])
        ax.plot(mean, label=f"{ylabel} (Random)",color=colors[0])
        ax.fill_between(range(len(mean)), mean-1.96*std, mean+1.96*std, alpha=0.2, color=colors[0])

    ax.set_xlabel("Chapter")
    ax.set_ylabel(f"{ylabel}")
    ax.set_title(f"{title} over time")

    # Set xticks
    n_chapters = {1 : 17, 2 : 18, 3 : 22, 4 : 37, 5 : 38, 6 : 30, 7 : 37}
    ax.set_xticks([0, *[sum(list(n_chapters.values())[:i]) for i in range(1,8)]], labels=[*[f"Book {i}" for i in range(1,8)], "End of Book 7"])
    plt.setp(ax.get_xticklabels(), rotation=30, horizontalalignment='right')

    ax.legend()
    
    plt.show()


n_chapters = {1 : 17, 2 : 18, 3 : 22, 4 : 37, 5 : 38, 6 : 30, 7 : 37}
combinations = list()
for book_nr in range(1,8):
    for chapter_nr in range(1,n_chapters[book_nr]+1):
        combinations.append((book_nr,chapter_nr))

random, cumulative, chapter = list(), list(), list()

centrality_cumulative, centrality_chapter = {char: list() for char in final_character_list.Name}, {char: list() for char in final_character_list.Name}

for comb in combinations:
    r, cu, cm, ccu, cch = get_overall_network_stats(comb[0],comb[1])

    random.append(r)
    cumulative.append(cu)
    chapter.append(cm)
    for char in final_character_list.Name:
        centrality_cumulative[char].append(ccu.get(char, 0))
        centrality_chapter[char].append(cch.get(char, 0))

for stat in cumulative[0].keys():
    if stat == "Modularity":
        plot_network_stat([x[stat] for x in cumulative], [x[stat] for x in chapter], None, stat, stat)
    else:
        plot_network_stat([x[stat] for x in cumulative], [x[stat] for x in chapter], [x[stat] for x in random], stat, stat)

c:\Users\rebek\miniconda3\envs\02467\lib\site-packages\networkx\algorithms\assortativity\correlation.py:298: RuntimeWarning: invalid value encountered in scalar divide
  return (xy * (M - ab)).sum() / np.sqrt(vara * varb)


display_chars = ['Harry Potter', "Severus Snape", 'Albus Dumbledore', "Lord Voldemort", "Hermione Granger","Ginny Weasley" ,"Ron Weasley", "Neville Longbottom"]
n_chapters = {1 : 17, 2 : 18, 3 : 22, 4 : 37, 5 : 38, 6 : 30, 7 : 37}

def plot_character_centrality_v1(display_chars, centrality_cumulative):
    fig, ax = plt.subplots(1, 1, figsize=(10,5))

    for i, char in enumerate(display_chars):
        ax.plot(centrality_cumulative[char], label=char, color=colors[i])

    ax.set_xticks([0, *[sum(list(n_chapters.values())[:i]) for i in range(1,8)]], labels = [*[f"Book {i}" for i in range(1,8)], "End of Book 7"])
    plt.setp(ax.get_xticklabels(), rotation=30, horizontalalignment='right')
    ax.set_xlabel("Chapter")
    ax.set_ylabel("Degree centrality")
    ax.set_title("Degree centrality over time")

    ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.25), ncol=4)

    plt.show()

def plot_character_centrality_v2(display_chars, centrality_cumulative, centrality_chapter):
    fig, ax = plt.subplots(4, 2, figsize=(12,8), sharex=True, sharey=True, constrained_layout=True)

    for i, char in enumerate(display_chars):
        axs = ax[i % int(len(display_chars)/2), i // int(len(display_chars)/2)]
        axs.plot(centrality_cumulative[char], label= "Accumulated", color=colors[0])
        axs.plot(centrality_chapter[char], label= "Chapter", color=colors[1])
        axs.set_xticks([])
        axs.set_title(char)

    for j in range(2):
        ax[3, j].set_xticks([0, *[sum(list(n_chapters.values())[:i]) for i in range(1,8)]], labels = [*[f"Book {i}" for i in range(1,8)], "End of Book 7"])
        plt.setp(ax[3, j].get_xticklabels(), rotation=30, horizontalalignment='right')
        ax[3, j].set_xlabel("Chapter")

    fig.legend(*ax[0,1].get_legend_handles_labels(), loc='upper right')
    
    for axs in ax[:,0]:
        axs.set_ylabel("Degree centrality")

    for axs in ax[:,1]:
        axs.set_yticks([])

    plt.show()


plot_character_centrality_v1(display_chars, centrality_cumulative)
plot_character_centrality_v2(display_chars, centrality_cumulative, centrality_chapter)


def get_community_matrix(previous, current):

    # Get number of communities
    N_previous = len(set(previous.values())) + 1
    N_current = len(set(current.values())) + 1

    # Get characters that are in both chapters
    characters = set(previous.keys() & current.keys())
    
    # Construct matrix
    M = np.zeros((N_previous, N_current))

    # Fill matrix
    for char in characters:
        if previous[char] != "Not present in previous chapter" and current[char] != "Not present in current chapter":
            M[previous[char], current[char]] += 1

    return M

def plot_community_matrix(M, title):
    fig, ax = plt.subplots(figsize=(10,10))
    ax.imshow(M, cmap = 'Reds')
    
    # plot options
    ax.set_xticks(np.arange(M.shape[1]))
    ax.set_yticks(np.arange(M.shape[0]))
    ax.set_xticklabels([f"Community {i}" for i in range(M.shape[1])])
    ax.set_yticklabels([f"Community {i}" for i in range(M.shape[0])]) 
    ax.set_title(title)
    plt.setp(ax.get_xticklabels(), rotation=30, ha="right", rotation_mode="anchor")
    plt.setp(ax.get_yticklabels(), rotation=30, ha="right", rotation_mode="anchor")

    for i in range(M.shape[0]):
        for j in range(M.shape[1]):
            c = M[i,j]
            ax.text(j, i, str(c), va='center', ha='center')
    
    plt.show()


def community_distribution(G1, G2, title1, title2):

    fig, ax = plt.subplots(1,2,figsize=(10,5))
    fig.suptitle("Community distribution")
    titles = [title1, title2]

    for i, G in enumerate([G1, G2]):
        community = [G.nodes[node]['group'] for node in G.nodes]
        ax[i].bar(range(0, max(community) + 1), [community.count(i) for i in range(max(community) + 1)], color = colors[i])
        
        # plot options
        ax[i].set_title(titles[i])
        ax[i].set_xlabel("Community")
        ax[i].set_ylabel("Frequency")
        ax[i].set_xticks(range(0, max(community) + 1), labels= [f"Community {i}" for i in range(max(community) + 1)])
        plt.setp(ax[i].get_xticklabels(), rotation=30, ha="right", rotation_mode="anchor")

    plt.show()


def get_central_nodes(G, top_n=10):

    centrality = nx.degree_centrality(G)
    central_nodes = sorted(centrality, key=centrality.get, reverse=True)[:top_n]

    return central_nodes


def get_central_nodes_community(G, part, top_n=5):

    centrality = nx.degree_centrality(G)
    
    central_nodes = dict()
    for community in set(part.values()):
        central_nodes[community] = sorted([node for node in centrality if part[node] == community], key=centrality.get, reverse=True)[:top_n]

    return central_nodes


def degree_distribution(G1, G2, title1, title2, N_bins = 50, log = True):

    fig, ax = plt.subplots(1,2,figsize = (15,5))
    fig.suptitle("Degree distribution")

    titles = [title1, title2]

    for i, G in enumerate([G1, G2]):
        degrees = [G.degree(node) for node in G.nodes]

        # Plot the distribution
        bins = np.logspace(0, np.log10(max(degrees)), N_bins)
        hist, edges = np.histogram(degrees, bins=bins, density=True)
        x = (edges[1:] + edges[:-1])/2
        x = x[hist>0]
        hist = hist[hist>0]

        ax[i].plot(x, hist, marker = ".", color = colors[i])
        ax[i].set_title(titles[i])
        ax[i].set_xlabel("Node Degree")
        ax[i].set_ylabel("Probability Density")

        if log:
            ax[i].set_xscale("log")
            ax[i].set_yscale("log")

    plt.show()


def degree_matrix(G1, G2, title1, title2):

    fig, ax = plt.subplots(1,2,figsize=(10,5))
    fig.suptitle("Degree Assortativity Matrix")
    titles = [title1, title2]

    for i, G in enumerate([G1, G2]):
        degree_matrix = nx.degree_mixing_matrix(G)

        ax[i].imshow(degree_matrix, cmap="BuGn")
        ax[i].set_title(titles[i])
        ax[i].set_xlabel("Degree")
        ax[i].set_ylabel("Degree")

        N = len(degree_matrix)
        ax[i].set_xticks(range(0,N,10))
        ax[i].set_xticklabels(range(1,N+1,10))
        ax[i].set_yticks(range(0,N,10))
        ax[i].set_yticklabels(range(1,N+1,10))

    plt.show()


def modularity_test(G1, G2, partition1, partition2, title1, title2, N_bins = 20):

    fig, ax = plt.subplots(1,2,figsize=(10,5))
    fig.suptitle("Modularity distribution of Swapped Graphs")
    titles = [title1, title2]

    for i, (G, partition) in enumerate(zip([G1, G2], [partition1, partition2])):
        modularity_values = ex_modularity(G, partition, n_swap=1000, experiments=100)

        bins = np.linspace(min(modularity_values), max(modularity_values), N_bins)
        hist, edges = np.histogram(modularity_values, bins=bins, density=True)
        x = (edges[1:] + edges[:-1])/2
        width = bins[1] - bins[0]

        ax[i].bar(x, hist, width=width, alpha=0.5, color = colors[0])
        ax[i].axvline(x=community_louvain.modularity(partition,G), color='black', linestyle='--', linewidth=2)
        ax[i].legend(['Original Modularity', 'Randomized Modularity'])
        ax[i].set_title(titles[i])
        ax[i].set_xlabel('Modularity')
        ax[i].set_ylabel('pdf')

    plt.show()


def get_key_moment_stats(pre: tuple = (1, 1, 4), post: tuple = (1, 1, 17), include_mod_degrees: bool = False):
    G_pre,_,partition_pre,_ = get_network_state(pre[0], pre[2], book_only = True, chapter_from=pre[1])
    G_post,_,partition_post,_ = get_network_state(post[0], post[2], book_only = True, chapter_from=pre[1])

    # Get central nodes
    central_nodes_pre = get_central_nodes(G_pre)
    central_nodes_post = get_central_nodes(G_post)

    # Get central nodes from each community
    central_nodes_community_pre = get_central_nodes_community(G_pre, partition_pre)
    central_nodes_community_post = get_central_nodes_community(G_post, partition_post)

    # Display community distribution
    community_distribution(G_pre, G_post, f"Community distribution chapter {pre[1]}-{pre[2]}", f"Community distribution chapter {post[1]}-{post[2]}")

    # Display communities change matrix
    M = get_community_matrix(partition_pre, partition_post)
    plot_community_matrix(M, f"Change in communites from chapter {pre[1]}-{pre[2]} to chapter {post[1]}-{post[2]}")

    # Display central nodes
    print(f"Central nodes of the graph of the first {pre[2]} chapters :")
    print(central_nodes_pre)

    print(f"Central nodes of the graph of the first {post[2]} chapters :")
    print(central_nodes_post)

    # Display central nodes from each community
    print(f"Central nodes from each community of the graph of the first {pre[2]} chapters :")
    for community in central_nodes_community_pre.keys():
        print(f"Community {community} : {central_nodes_community_pre[community]}")


    print(f"Central nodes from each community of the graph of the first {post[2]} chapters :")
    for community in central_nodes_community_post.keys():
        print(f"Community {community} : {central_nodes_community_post[community]}")
    
    if include_mod_degrees:
        title1, title2 = f"Chapters {pre[1]}-{pre[2]}", f"Chapters {post[1]}-{post[2]}"
        modularity_test(G_pre, G_post, partition_pre, partition_post, title1=title1, title2=title2)
        degree_distribution(G_pre, G_post, title1=title1, title2=title2)
        degree_matrix(G_pre, G_post, title1=title1, title2=title2)


# Muggles vs Wizards
get_key_moment_stats((1,1,4), (1,1,17), include_mod_degrees=True)

Central nodes of the graph of the first 4 chapters :
['Harry Potter', 'Rubeus Hagrid', 'Dudley Dursley', 'Vernon Dursley', 'Petunia Dursley', 'Three Muggle boys', 'Stan Shunpike', 'Albus Dumbledore', 'Gringotts Head Goblin', 'Lord Voldemort']
Central nodes of the graph of the first 17 chapters :
['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Rubeus Hagrid', 'Albus Dumbledore', 'Minerva McGonagall', 'Neville Longbottom', 'Draco Malfoy', 'Madam Malkin', 'Severus Snape']
Central nodes from each community of the graph of the first 4 chapters :
Community 0 : ['Rubeus Hagrid', 'Albus Dumbledore', 'Gringotts Head Goblin', 'Lord Voldemort', 'Wizard who claimed to be a dragon killer']
Community 1 : ['Three Muggle boys', 'Stan Shunpike', 'Emeric Switch', 'Quentin Trimble', 'Phyllida Spore']
Community 2 : ['Harry Potter', 'Dudley Dursley', 'Vernon Dursley', 'Petunia Dursley', 'Arabella Figg']
Central nodes from each community of the graph of the first 17 chapters :
Community 0 : ['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Rubeus Hagrid', 'Albus Dumbledore']
Community 1 : ['Three Muggle boys', 'Magical Menagerie saleswoman', 'Stan Shunpike', 'Emeric Switch', 'Cuthbert Binns']
Community 2 : ['Dudley Dursley', 'Vernon Dursley', 'Petunia Dursley', 'Fat Friar', 'Lavender Brown']
Community 3 : ['Dean Thomas', 'Cho Chang', 'Black Lake', 'Thomas Riddle', 'Sorting Hat']
Community 4 : ['Madam Malkin', 'George Weasley', 'Fred Weasley', 'Oliver Wood', 'Wizard who claimed to be a dragon killer']

100%|██████████| 100/100 [00:15<00:00,  6.64it/s]
100%|██████████| 100/100 [01:48<00:00,  1.08s/it]


# Dumbledore's Army
get_key_moment_stats((5,1,17), (5,18,38), include_mod_degrees=True)

Central nodes of the graph of the first 17 chapters :
['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Albus Dumbledore', 'Sirius Black', 'Fred Weasley', 'Dolores Umbridge', 'George Weasley', 'Ginny Weasley', 'Neville Longbottom']
Central nodes of the graph of the first 38 chapters :
['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Albus Dumbledore', 'Dolores Umbridge', 'Sirius Black', 'Neville Longbottom', 'George Weasley', 'Fred Weasley', 'Ginny Weasley']
Central nodes from each community of the graph of the first 17 chapters :
Community 0 : ['Sirius Black', 'Remus Lupin', 'Wizard who claimed to be a dragon killer', 'Molly Weasley', 'Arthur Weasley']
Community 1 : ['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Albus Dumbledore', 'Dolores Umbridge']
Community 2 : ['Draco Malfoy', 'Gregory Goyle', 'Pansy Parkinson', 'Vincent Crabbe', 'Stan Shunpike']
Community 3 : ['Fred Weasley', 'George Weasley', 'Ginny Weasley', 'Neville Longbottom', 'Cho Chang']
Community 4 : ['Dudley Dursley', 'Vernon Dursley', 'Three Muggle boys', 'Petunia Dursley', 'Arabella Figg']
Central nodes from each community of the graph of the first 38 chapters :
Community 0 : ['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Albus Dumbledore', 'Dolores Umbridge']
Community 1 : ['George Weasley', 'Fred Weasley', 'Draco Malfoy', 'Ali Bashir', 'Gregory Goyle']
Community 2 : ['Sirius Black', 'Remus Lupin', 'Molly Weasley', 'Black Lake', 'Arthur Weasley']
Community 3 : ['Neville Longbottom', 'Ginny Weasley', 'Luna Lovegood', 'Cho Chang', 'Dean Thomas']
Community 4 : ['Madam Malkin', 'Dudley Dursley', 'Poppy Pomfrey', 'Vernon Dursley', 'Petunia Dursley']

100%|██████████| 100/100 [03:11<00:00,  1.91s/it]
100%|██████████| 100/100 [05:16<00:00,  3.16s/it]


# Snape's True Allegiance
get_key_moment_stats((6,20,25), (6,26,30), include_mod_degrees=False)
get_key_moment_stats((7,1,32), (7,33,37), include_mod_degrees=False)

Central nodes of the graph of the first 25 chapters :
['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Albus Dumbledore', 'Severus Snape', 'Horace Slughorn', 'Draco Malfoy', 'Ginny Weasley', 'Lavender Brown', 'Rubeus Hagrid']
Central nodes of the graph of the first 30 chapters :
['Harry Potter', 'Albus Dumbledore', 'Ron Weasley', 'Hermione Granger', 'Severus Snape', 'Ginny Weasley', 'Draco Malfoy', 'Horace Slughorn', 'Minerva McGonagall', 'Rubeus Hagrid']
Central nodes from each community of the graph of the first 25 chapters :
Community 0 : ['Harry Potter', 'Albus Dumbledore', 'Horace Slughorn', 'Rubeus Hagrid', 'Sirius Black']
Community 1 : ['Cho Chang', 'Remus Lupin', 'James Potter', 'Peter Pettigrew', 'Lily Potter']
Community 2 : ['Ron Weasley', 'Hermione Granger', 'Severus Snape', 'Draco Malfoy', 'Inferi Army of Lord Voldemort']
Community 3 : ['Ginny Weasley', 'Lavender Brown', 'Dean Thomas', 'Seamus Finnigan', 'Mungo Bonham']
Central nodes from each community of the graph of the first 30 chapters :
Community 0 : ['Minerva McGonagall', 'Remus Lupin', 'Bill Weasley', 'Madam Malkin', 'Argus Filch']
Community 1 : ['Rufus Scrimgeour', 'Inferi Army of Lord Voldemort', 'Vincent Crabbe', 'Gregory Goyle', 'Percy Weasley']
Community 2 : ['Harry Potter', 'Albus Dumbledore', 'Severus Snape', 'Draco Malfoy', 'Horace Slughorn']
Community 3 : ['Ron Weasley', 'Hermione Granger', 'Ginny Weasley', 'Neville Longbottom', 'Lavender Brown']

Central nodes of the graph of the first 32 chapters :
['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Albus Dumbledore', 'Death', 'Lord Voldemort', 'Severus Snape', 'Remus Lupin', 'Wizard who claimed to be a dragon killer', 'Rubeus Hagrid']
Central nodes of the graph of the first 37 chapters :
['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Albus Dumbledore', 'Death', 'Lord Voldemort', 'Remus Lupin', 'Ginny Weasley', 'Severus Snape', 'Luna Lovegood']
Central nodes from each community of the graph of the first 32 chapters :
Community 0 : ['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Death', 'Luna Lovegood']
Community 1 : ['Neville Longbottom', 'Minerva McGonagall', 'Cho Chang', 'Amycus Carrow', 'Michael Corner']
Community 2 : ['Loxias', 'Emeric Switch', 'Godelot', 'Livius', 'Hereward']
Community 3 : ['Albus Dumbledore', 'Severus Snape', 'Wizard who claimed to be a dragon killer', 'Sirius Black', 'Godric Gryffindor']
Community 4 : ['Lord Voldemort', 'Griphook', 'Dean Thomas', 'Draco Malfoy', 'Bellatrix Lestrange']
Community 5 : ['Remus Lupin', 'Rubeus Hagrid', 'Fleur Delacour', 'Fred Weasley', 'Ginny Weasley']
Community 6 : ['Vernon Dursley', 'Petunia Dursley', 'Dudley Dursley', 'Hedwig', 'Dedalus Diggle']
Central nodes from each community of the graph of the first 37 chapters :
Community 0 : ['Vernon Dursley', 'Petunia Dursley', 'Dudley Dursley', 'Hedwig', 'Barny Weasley']
Community 1 : ['Loxias', 'Emeric Switch', 'Godelot', 'Livius', 'Hereward']
Community 2 : ['Lord Voldemort', 'Draco Malfoy', 'Bellatrix Lestrange', 'Griphook', 'Fenrir Greyback']
Community 3 : ['Albus Dumbledore', 'Severus Snape', 'Wizard who claimed to be a dragon killer', 'Aberforth Dumbledore', 'Sirius Black']
Community 4 : ['Harry Potter', 'Ron Weasley', 'Hermione Granger', 'Death', 'Xenophilius Lovegood']
Community 5 : ['Remus Lupin', 'Ginny Weasley', 'Rubeus Hagrid', 'George Weasley', 'Fred Weasley']
Community 6 : ['Luna Lovegood', 'Neville Longbottom', 'Minerva McGonagall', 'Horace Slughorn', 'Cho Chang']


# Get the vocabulary of all characters
books = {nr: [] for nr in range(1,8)}
for book_nr in range(1,8):
        # Load formatted book (dict of chapters with list of lists of pages and sentences)
    with open(f"Data/Books formatted/Book{book_nr}.pkl","rb") as f:
        book = pkl.load(f)

    for chapter in book.values():
        for page in chapter:
            books[book_nr].extend([token for sent in page for token in tokenize(sent)])

def get_text(start: list = [1, 1], stop: list = [7, 37], characters_to_get: list = [], alltexts: dict = book_text) -> dict:
    
    text_dict = {name: [] for name in characters_to_get}
    
    for char in characters_to_get:
        temp = {booknr: alltexts[char].get(booknr) for booknr in range(start[0], stop[0]+1)}
        for booknr in range(start[0], stop[0]+1):
            if start[0] == stop[0]:
                # add the text from the start to stop chapters of that one book 
                text_dict[char].append([text['text'] for text in itemgetter(*range(start[1], stop[1]+1))(temp[booknr])])
            
            else: 
                if booknr == start[0]:
                    # add the text from the start book starting from the start chapter to the end
                    text_dict[char].append([text['text'] for text in itemgetter(*range(start[1], len(temp[booknr])+1))(temp[booknr])])
                elif booknr == stop[0]:
                    # add the text from the stop book starting from the start to the stop chapter
                    text_dict[char].append([text['text'] for text in itemgetter(*range(1, stop[1]+1))(temp[booknr])])
                else:
                    # add the text from the middle books completely
                    text_dict[char].append([text['text'] for text in temp[booknr].values()])
    return text_dict

all_words = set()
for book in books.values():
        all_words = all_words.union(book)

# Get document term count (document = chapter)
def get_doc_term_count(documents: dict = book_text, all_terms: set = all_words):
    doc_term_count = {}
    for term in tqdm(all_terms):
        for char in documents.keys():
            for book in documents[char].values():
                doc_term_count.update({term: doc_term_count.get(term, 0) + sum([1 for chapter in book.values() if term in chapter['tokens']])})

    return doc_term_count

doc_term_count = get_doc_term_count()

# Get document term count (document = book)
def get_doc_term_count_books(documents: dict = books, all_terms: set = all_words):
    doc_term_count_book = {}
    for term in tqdm(all_terms):
        doc_term_count_book.update({term: sum([1 for book in books.values() if term in book])})

    return doc_term_count_book

doc_term_count_book = get_doc_term_count_books()


# load the all_word set ad doc_term_count
with open('Temp/tf_idf_support.pkl', 'rb') as f:
    all_words, doc_term_count, doc_term_count_book = pkl.load(f)


# Calculate tf
def calc_tf(character: str = "", position: Union[list, int] = [1,1], documents: tuple = (book_text, books), cumulative: bool = False):
    if cumulative:
        documents = documents[1]
        terms = documents[position]
    else:
        documents = documents[0]
        terms = documents[character][position[0]][position[1]]['tokens']
    
    frequency = nltk.FreqDist(terms)

    sum_terms = len(terms)
    tf = {word: freq/sum_terms for word, freq in frequency.items() if word not in [el.lower() for char in final_character_list.Name for el in char.split(' ')] + ['said']}

    return tf

# Calculate idf
def calc_idf(position: Union[list, int] = [1,1], documents: tuple = (book_text, books), all_terms: set = all_words,
            doc_term: tuple = (doc_term_count, doc_term_count_book), cumulative: bool = False):
    if cumulative:
        documents = documents[1]
        doc_term = doc_term[1]
        N = len(documents)
    else:
        documents = documents[0]
        doc_term = doc_term[0]
        N = sum([len(documents['Harry Potter'][i]) for i in range(1,8)])*len([char for char in final_character_list.Name])

    idf = {word: np.log2(N/(1 + doc_term[word])) for word in all_terms}

    return idf

# Calculate tf-idf
def calc_tf_idf(character: str = "", position: Union[list, int] = [1,1], cumulative: bool = False):
    if cumulative:
        tf = calc_tf(position = position, cumulative = True)
        idf = calc_idf(position = position, cumulative = True)
    else:
        tf = calc_tf(character = character, position = position, cumulative = False)
        idf = calc_idf(position = position, cumulative = False)

    tf_idf = {word: tf[word] * idf[word] for word in tf.keys()}

    return tf_idf


def plot_wordcloud(characters_to_get: list = [], position: list = [1,1], mask: list = None,
                    colormap: list = ['OrRd', 'Blues', 'Greens', 'Purples', 'Reds']):
    fig, ax = plt.subplots(1, len(characters_to_get), figsize=(15,20), constrained_layout=True)
    for i, char in enumerate(characters_to_get):
        if mask is not None:
            mask1 = np.array(Image.open(mask[i]))
            mask1 = np.where(mask1==0, 255, mask1)
            wordcloud = WordCloud(background_color="white",
                                mask=mask1,
                                contour_color='black',
                                contour_width=1,
                                colormap=colormap[i]
                                ).generate_from_frequencies(calc_tf_idf(character=char, position=position))
        else:
            wordcloud = WordCloud(background_color="white",
                                    colormap=colormap[i]
                                    ).generate_from_frequencies(calc_tf_idf(character=char, position=position))
        
        ax[i].imshow(wordcloud, interpolation='bilinear')
        ax[i].set_title(f"{char} \n in book {position[0]}, chapter {position[1]}", fontsize=14)
        ax[i].axis("off")
    plt.show()

def plot_wordcloud_book(mask: list = None, colormap: list = ["OrRd"]):

    for booknr in range(1,8):
        fig, ax = plt.subplots(1, 1, figsize=(10,10))
        if mask is not None:
            mask1 = np.array(Image.open(mask[booknr-1]))
            mask1 = np.where(mask1==0, 255, mask1)
            wordcloud = WordCloud(background_color="white",
                                mask=mask1,
                                contour_color='black',
                                contour_width=1,
                                colormap=colormap[booknr-1]
                                ).generate_from_frequencies(calc_tf_idf(position = booknr, cumulative = True))
        else:
            wordcloud = WordCloud(background_color="white",
                                    colormap=colormap[booknr-1]
                                    ).generate_from_frequencies(calc_tf_idf(position = booknr, cumulative = True))
        
        ax.imshow(wordcloud, interpolation='bilinear')
        ax.axis("off")
        plt.show()

plot_wordcloud_book(mask=['3.Text_files/WordCloud_style/book1.png',
                           '3.Text_files/WordCloud_style/book2.png',
                           '3.Text_files/WordCloud_style/book3.png',
                           '3.Text_files/WordCloud_style/book4.png',
                           '3.Text_files/WordCloud_style/book5_1.png',
                           '3.Text_files/WordCloud_style/book6.png',
                           '3.Text_files/WordCloud_style/book7.png'],
                    colormap=['OrRd', 'BuGn', 'twilight', 'PuBu', 'Wistia', 'BrBG', 'bone'])


def get_flesch_reading_ease(text: list) -> float:
    """
    Returns the Flesch reading ease score for a given text.
    
    Input is a list of sentences.
    """
    n_words = sum([len(sent.split(' ')) for sent in text])
    n_sents = len(text)

    hyphenator = pyphen.Pyphen(lang='en_UK')
    n_syllables = 0
    for sent in text:
        sent = sent.lower()
        n_syllables += len([string for string in re.split('-|\s', hyphenator.inserted(sent)) if string != ''])

    return 206.835 - 1.015*(n_words/n_sents) - 84.6*(n_syllables/n_words)

flesch_reading_ease = []

for book_nr in range(1,8):
    with open(f"1.Dataset_files/Books_formatted/Book{book_nr}.pkl","rb") as f:
        book = pkl.load(f)

    booktext = []
    for chapternr in book.keys():
        booktext.extend([sent for page in book[chapternr] for sent in page])

    flesch_reading_ease.append(get_flesch_reading_ease(booktext))

fig, axs = plt.subplots(1,1, figsize=(15, 6))

plt.plot(range(1,8), flesch_reading_ease, color = 'k', label = "Flesch reading ease score")

# Flesch reading ease context
plt.axhline(100, color='lightsteelblue', linestyle='--')
plt.axhline(90, color='cornflowerblue', linestyle='--')
plt.axhline(80, color='royalblue', linestyle='--')
plt.axhline(70, color='mediumblue', linestyle='--')
plt.axhline(60, color='darkblue', linestyle='--')

# text labels
plt.text(1, 100.2, "Very easy to read (5th grade level)", fontsize=12, color='lightsteelblue')
plt.text(1, 90.2, "Easy to read (6th grade level)", fontsize=12, color='cornflowerblue')
plt.text(1, 80.2, "Fairly easy to read (7th grade level)", fontsize=12, color='royalblue')
plt.text(1, 70.2, "Plain English (8th & 9th grade level)", fontsize=12, color='mediumblue')
plt.text(1, 60.2, "Fairly difficult to read (10th to 12th grade level)", fontsize=12, color='darkblue')

plt.title("Flesch reading ease for each book")
plt.xlabel("Book")
plt.ylabel("Flesch reading ease score")
plt.show()


# Get the sentiment of text
def get_sentiment(text: dict) -> dict:

    sid = SentimentIntensityAnalyzer()
    sentiment_mean = {name: [] for name in text.keys()}
    sentiment_se = {name: [] for name in text.keys()}
    
    for char in text.keys():
        for book in text[char]:
            for chapter in book:
                if chapter == []:
                    sentiment_mean[char].append(np.nan)
                    sentiment_se[char].append(np.nan)
                else:
                    chapter_sentiments = []
                    for sentence in chapter:
                        chapter_sentiments.append(sid.polarity_scores(sentence)['compound'])

                    sentiment_mean[char].append(np.nanmean(chapter_sentiments))
                    sentiment_se[char].append(np.nanstd(chapter_sentiments)/np.sqrt(len(chapter_sentiments)))

    return sentiment_mean, sentiment_se

def plot_sentiment(start: list = [1, 1], stop: list = [7, 37], characters_to_get: list = [],
colors: list = colors):
    """
    Plots the sentiment of the characters in the window between start and stop.
    """
    fig, ax = plt.subplots(len(characters_to_get), 1,
                            figsize = (25, 20), constrained_layout=True, sharex=True)

    text_dict = get_text(start, stop, characters_to_get)
    sentiment_mean, sentiment_se = get_sentiment(text_dict)
    for i, (char, sentiment) in enumerate(sentiment_mean.items()):
        ax[i].plot(sentiment, label=char, color = colors[i])
        ax[i].fill_between(range(len(sentiment)), sentiment - 1.96*np.array(sentiment_se[char]), sentiment + 1.96*np.array(sentiment_se[char]), alpha = 0.5, color = colors[i])
        ax[i].axhline(0, color='grey', linestyle='--')
        ax[i].set_xlim(0, len(sentiment_mean[characters_to_get[0]])-1)
        ax[i].set_ylim(-1, 1)
        ax[i].set_title(f"{char}")

    chapters = [17, 18, 22, 37, 38, 30, 37]

    # Create xlabel for the plot in the form of "Book x, Chapter x"
    xlabel = []
    curr_book = start[0]
    curr_chapter = start[1]

    for i in range(len(sentiment_mean[characters_to_get[0]])):
        if curr_chapter <= chapters[curr_book-1]:
            xlabel.append(f"Book {curr_book}, Chapter {curr_chapter}")
            curr_chapter += 1
        else:
            curr_book += 1
            curr_chapter = 1
            xlabel.append(f"Book {curr_book}, Chapter {curr_chapter}")
            curr_chapter += 1

    ax[-1].set_xticks(range(len(sentiment_mean[characters_to_get[0]])), xlabel, rotation=90)
    ax[-1].set_ylabel("Sentiment")
    ax[-1].set_xlabel("Book, Chapter")
    fig.suptitle(f"Sentiment of characters in book {start[0]} chapter {start[1]} and book {stop[0]} chapter {stop[1]}", fontsize=16)

    plt.show()


# Muggles vs. Wizards
key_characters = ['Harry Potter', 'Rubeus Hagrid', 'Dudley Dursley', 'Albus Dumbledore', 'Quirinus Quirrell']

plot_sentiment(start=[1,1], stop=[1, 17], characters_to_get=key_characters, colors = ['darkred', 'darkslateblue', 'darkgreen', 'darkorange', 'darkorchid'])
plot_wordcloud(characters_to_get=['Harry Potter', 'Rubeus Hagrid', 'Dudley Dursley', 'Albus Dumbledore'], position=[1,4], colormap=['OrRd', 'Blues', 'Greens', 'copper', 'bone'])
plot_wordcloud(characters_to_get=['Harry Potter', 'Rubeus Hagrid', 'Albus Dumbledore', 'Quirinus Quirrell'], position=[1,17], colormap=['OrRd', 'Blues', 'copper', 'bone'])


# Dumbledore's Army
key_characters = ["Harry Potter", "Neville Longbottom", "Ginny Weasley", "Draco Malfoy", "Cho Chang", "Dolores Umbridge"]

plot_sentiment(start=[5, 15], stop=[5, 38], characters_to_get=key_characters, colors = ['darkred', 'crimson', 'orangered', 'darkgreen', 'darkslateblue', 'tab:pink'])
plot_wordcloud(characters_to_get=key_characters, position=[5, 17], colormap=['OrRd', 'Reds', 'Oranges', 'Greens', 'Blues', 'RdPu'])
plot_wordcloud(characters_to_get=key_characters, position=[5, 29], colormap=['OrRd', 'Reds', 'Oranges', 'Greens', 'Blues', 'RdPu'])


# Snape's true alliegance
key_characters = ["Harry Potter", "Severus Snape", "Albus Dumbledore", "Lord Voldemort", "Lily Potter"]
cmaps = ['OrRd', 'Greens', 'Blues', 'bone', 'Reds']

plot_sentiment(start=[6, 26], stop=[7, 37], characters_to_get=key_characters, colors = ['darkred', 'darkgreen', 'darkslateblue', 'mediumseagreen', 'crimson'])
plot_wordcloud(characters_to_get=key_characters[:4], position=[6, 26], colormap=cmaps)
plot_wordcloud(characters_to_get=key_characters, position=[7, 33], colormap=cmaps)

Motivation¶

What is your dataset?¶

Why did you choose these particular datasets?¶

What was your goal for the end user's experience?¶

Basic Stats¶

Data cleaning and preprocessing choices¶

Data cleaning of the books and character lists¶

Pre-processing to set up the dynamic chapter networks¶

Preprocessing to set up the dynamic character text¶

Limitations to data cleaning and preprocessing choices¶

Final dataset stats¶

Network statistics¶

Dynamic text statistics¶

Tools, theory and analysis¶

Tools and theory for Network analysis¶

Getting the Character Network at a specific point in the book¶

Complete expanding network analysis¶

Centrality of main characters¶

Walktrough of the network analysis methods in the key moments¶

Comparing changes in the community structure of the Character Network¶

Distribution of the Community Structure¶

Central Characters¶

Distribution of node degrees and interactions between characters compared to character significance¶

Accessing significance of the network partitioning¶

Calling the tools on different key moments¶

Visualizing the networks¶

Network results¶

Tools and theory for Text analysis¶

Getting the dynamic character text for a specific character¶

TF-IDF¶

Pre-processing for TF-IDF¶

Implementation of TF-IDF¶

Implementation of Wordclouds¶

Flesch Reading Ease¶

Sentiment Analysis¶

Text Key Moments Results¶

Analysis big picture¶

Why do the chosen tools answer our research questions?¶

What are the limitations of the analysis?¶

Discussion¶

What went well?¶

What is still missing? What could be improved? Why?¶

Suggestions for further research¶